package spider.hysrlzy;

import com.yc.favorite.util.MyBatisHelper;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import spider.LocalRes;
import spider.SpiderMain;
import spider.hysrlzy.mapper.CategroyDao;
import spider.hysrlzy.mapper.JobMapper;
import spider.hysrlzy.mapper.UserMapper;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;

public class HysrlzyMain {

    public static void main(String[] args) throws IOException {
        //capture4();
        LocalRes.loacal("http://www.hysrlzy.com/",
                "D:/_文件柜/94/0723/hr/index.html");
        getOtherPageRes();
    }


    /*
     * 获取主页中的其他页面的信息
     */
    public static void getOtherPageRes() throws IOException {
        long startTime = System.currentTimeMillis();
        //获取首页中的其他页面的资源
//        Map<String,String> urlMap= LocalRes.getUrl("http://www.hysrlzy.com/");
        List<String> list = LocalRes.getUrl2("http://www.hysrlzy.com/");
//        for (Map.Entry<String,String> m : urlMap.entrySet()){
//            list.add(m.getValue());
//        }
        System.out.println(list.size());
        Thread b = new Thread(){
            public void run(){
                for (int i = 0; i < list.size()/2; i++) {
                    try {
                        LocalRes.local2(list.get(i),"G:/Web/hysrlzy/");
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        };
        Thread a = new Thread(){
            public void run(){
                for (int i = list.size()-1; i >= list.size()/2; i--) {
                    try {
                        LocalRes.local2(list.get(i),"G:/Web/hysrlzy/");
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        };
        a.start();
        b.start();
        try {
            a.join();
            b.join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

//        for (Map.Entry<String,String> m : urlMap.entrySet()){
//            System.out.println("下载开始:"+m.getKey());
//            LocalRes.local(m.getKey(),"G:/Web/hysrlzy/");
//        }

        System.out.println("耗时:"+(System.currentTimeMillis()-startTime)/1000+"秒");
        System.out.println("完成");

    }



    /**
     * 爬取用户分页数据
     */
    static void capture4() throws IOException {
        SqlSession session = MyBatisHelper.openSession();
        UserMapper um = session.getMapper(UserMapper.class);
        /**
         * 爬取分页数据
         */
        int page = 1;
        while (true) {
            // 设置 baseUrl
            String url = "http://www.hysrlzy.com/Jianli";
            Document doc = Jsoup.parse(new URL(url + "/new.aspx?PageNo=" + page), 5000);
            if (doc.select(".seaList>.seaLists").size() == 0) {
                break;
            }
            SpiderMain sm = new SpiderMain(
                    url,
                    new UserUrlCapturer(), new UserDataCapturer(), um);
            // 添加真正要爬取的页面地址
            sm.getUrlSet().add(url + "/new.aspx?PageNo=" + page);
            sm.start();
            session.commit();
            page++;
        }
    }

    // 爬取分类数据
    static void capture3() throws IOException {
        SpiderMain sm = new SpiderMain(
                "http://www.hysrlzy.com/jobseek/class.aspx/",
                null,
                new CategoryDataCapturer(),
                new CategroyDao());
        sm.start();
    }

    /**
     * 爬取分页数据
     */
    static void capture2() throws IOException {
        SqlSession session = MyBatisHelper.openSession();
        JobMapper jm = session.getMapper(JobMapper.class);
        /**
         * 爬取分页数据
         */
        // http://www.hysrlzy.com/job/?PageNo=2
        int page = 1;
        while (true) {
            // 判断该页是否由数据, 如果没有则退出分页爬取
            String url = "http://www.hysrlzy.com/job/?PageNo=" + page;
            Document doc = Jsoup.parse(new URL(url), 5000);
            if (doc.select(".seaList>.seaLists").size() == 0) {
                break;
            }
            SpiderMain sm = new SpiderMain(
                    "http://www.hysrlzy.com/job/?PageNo=" + page,
                    new JobUrlCapturer(), new JobDataCapturer(), jm);
            sm.start();
            session.commit();
            page++;
        }
    }

    /**
     * 爬取首页的最新职位
     */
    static void capture1() {
        SqlSession session = MyBatisHelper.openSession();
        JobMapper jm = session.getMapper(JobMapper.class);
        SpiderMain sm = new SpiderMain(
                "http://www.hysrlzy.com/jobseek/",
                new JobUrlCapturer(), new JobDataCapturer(), jm);
        sm.start();
        session.commit();
    }

}
